BOT_NAME = 'tutorial'

SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'

# 配置 Selenium 中间件
DOWNLOADER_MIDDLEWARES = {
    'tutorial.middlewares.SeleniumMiddleware': 543,
}

# 设置爬虫速度，避免被封
DOWNLOAD_DELAY = 2
AUTOTHROTTLE_ENABLED = True

# 设置 Selenium 选项
SELENIUM_DRIVER_NAME = 'chrome'
SELENIUM_DRIVER_EXECUTABLE_PATH = 'C:\\Users\\apps\\chromedriver-win64\\chromedriver.exe'  # chrome``driver 路径
SELENIUM_DRIVER_ARGUMENTS = [
    # '--headless',  # 无头模式
    '--no-sandbox',
    '--disable-dev-shm-usage',
    '--disable-gpu',
    '--window-size=1920,1080'
]

# 启用管道
ITEM_PIPELINES = {
    'tutorial.pipelines.TutorialPipeline': 300,
}

# 日志级别
LOG_LEVEL = 'INFO'

# 用户代理池
USER_AGENT_LIST = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1'
]